from gxl_ai_utils.utils import utils_file

def get_full_labeled_data():
    """"""
    dir_path = "/teaspeech_ceph/share_976139/users/xuelonggeng/code/ft_local/emotion2vec/tmp_res"
    jsonl_path_list = utils_file.do_get_list_for_wav_dir(dir_path, suffix=".jsonl")
    utils_file.print_list(jsonl_path_list)
    new_res_list = []
    for jsonl_path in utils_file.tqdm(jsonl_path_list, total=len(jsonl_path_list), desc="get_full_labeled_data"):
        res_list = utils_file.load_dict_list_from_jsonl(jsonl_path)
        new_res_list.extend(res_list)
    new_res_keys = set([item["key"] for item in new_res_list])
    # 保存结果到文件
    output_path = "/teaspeech_ceph/share_976139/users/xuelonggeng/data/asr_6.4w/score_all_6.4w_full_info_with_wav_score_up_8_with_emotion2vec_label_part1_all2.jsonl"
    utils_file.write_dict_list_to_jsonl(new_res_list, output_path)
    part2_output_path = "/teaspeech_ceph/share_976139/users/xuelonggeng/data/asr_6.4w/score_all_6.4w_full_info_with_wav_score_up_8_part2_all2.jsonl"
    up8_path = "/teaspeech_ceph/share_976139/users/xuelonggeng/data/asr_6.4w/score_all_6.4w_full_info_with_wav_score_up_8.jsonl"
    up8_all_list = utils_file.load_dict_list_from_jsonl(up8_path)
    part2_list = []
    for dict_i in utils_file.tqdm(up8_all_list, total=len(up8_all_list), desc="get_full_labeled_data part2"):
        if dict_i["key"] not in new_res_keys:
            part2_list.append(dict_i)
    utils_file.write_dict_list_to_jsonl(part2_list, part2_output_path)


    # new_dict_list_only_valid = []
    # exclude_labels = ("中立/neutral", "其他/other", "<unk>")
    # for dict_i in utils_file.tqdm(new_res_list, total=len(new_res_list), desc="filter valid data"):
    #     if dict_i["is_valid"] or dict_i["emotion"] not in exclude_labels:
    #         new_dict_list_only_valid.append(dict_i)
    # output_path = "/teaspeech_ceph/share_976139/users/xuelonggeng/data/asr_6.4w/score_all_6.4w_full_info_with_wav_score_up_8_with_emotion2vec_label_only_valid.jsonl"
    # utils_file.write_dict_list_to_jsonl(new_dict_list_only_valid, output_path)

if __name__ == "__main__":
    get_full_labeled_data()